{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'mesolitica-storage.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "client = storage.Client()\n",
    "bucket = client.bucket('mesolitica-general')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm t5-base-summary/*\n",
    "\n",
    "model = '537600'\n",
    "\n",
    "blob = bucket.blob(f't5-base-generator-v1/model.ckpt-{model}.data-00000-of-00002')\n",
    "blob.download_to_filename(f't5-base-summary/model.ckpt-{model}.data-00000-of-00002')\n",
    "\n",
    "blob = bucket.blob(f't5-base-generator-v1/model.ckpt-{model}.data-00001-of-00002')\n",
    "blob.download_to_filename(f't5-base-summary/model.ckpt-{model}.data-00001-of-00002')\n",
    "\n",
    "blob = bucket.blob(f't5-base-generator-v1/model.ckpt-{model}.index')\n",
    "blob.download_to_filename(f't5-base-summary/model.ckpt-{model}.index')\n",
    "\n",
    "blob = bucket.blob(f't5-base-generator-v1/model.ckpt-{model}.meta')\n",
    "blob.download_to_filename(f't5-base-summary/model.ckpt-{model}.meta')\n",
    "\n",
    "blob = bucket.blob('t5-base-generator-v1/checkpoint')\n",
    "blob.download_to_filename('t5-base-summary/checkpoint')\n",
    "\n",
    "blob = bucket.blob('t5-base-generator-v1/operative_config.gin')\n",
    "blob.download_to_filename('t5-base-summary/operative_config.gin')\n",
    "\n",
    "with open('t5-base-summary/checkpoint', 'w') as fopen:\n",
    "    fopen.write(f'model_checkpoint_path: \"model.ckpt-{model}\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "import t5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = t5.models.MtfModel(\n",
    "    model_dir='t5-base-summary',\n",
    "    tpu=None,\n",
    "    tpu_topology=None,\n",
    "    model_parallelism=2,\n",
    "    batch_size=1,\n",
    "    sequence_length={\"inputs\": 1024, \"targets\": 1024},\n",
    "    learning_rate_schedule=0.003,\n",
    "    save_checkpoints_steps=5000,\n",
    "    keep_checkpoint_max=3,\n",
    "    iterations_per_loop=100,\n",
    "    mesh_shape=\"model:1,batch:1\", \n",
    "    mesh_devices=[\"cpu:0\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = '1. Dr M perlu dikekalkan sebagai perdana menteri. 2. Muhyiddin perlulah menolong Dr M. 3. rakyat perlu menolong Muhyiddin.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = '1. kerajaan perlu tolong gotong royong. 2. masyarakat juga perlu menolong kerajaan. 3. ibu bapa perlu memastikan anak menolong kerajaan'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = '1. Neelofa tetap dengan keputusan untuk berkahwin akhir tahun ini. 2. Long Tiger sanggup membantu Neelofa. 3. Tiba-tiba Long Tiger bergaduh dengan Husein.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = '1. menambahkan ilmu tentang tempat yang dilawati. 2. memanfaatkan masa yang terluang ke tempat yang berfaedah. 3. menambahkan semangat kecintaan negara. 4. memberikan hiburan - sambil melawat sambil berhibur dan berseronok'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. Dr M perlu dikekalkan sebagai perdana menteri. 2. Muhyiddin perlulah menolong Dr M. 3. rakyat perlu menolong Muhyiddin.\n"
     ]
    }
   ],
   "source": [
    "print(string)\n",
    "with tf.io.gfile.GFile('test.txt', \"w\") as f:\n",
    "    f.write(\"karangan: %s\\n\" % string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_model_dir': 't5-base-summary', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "isolate_session_state: true\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': None, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver.TPUClusterResolver object at 0x7fc9483795c0>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running infer on CPU\n",
      "INFO:tensorflow:feature inputs : Tensor(\"Reshape:0\", shape=(1, 1, 1024), dtype=int32)\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/mesh_tensorflow/transformer/utils.py:386: Print (from tensorflow.python.ops.logging_ops) is deprecated and will be removed after 2018-08-20.\n",
      "Instructions for updating:\n",
      "Use tf.print instead of tf.Print. Note that tf.print returns a no-output operator that directly prints the output. Outside of defuns or eager mode, this operator will not be executed unless it is directly specified in session.run or used as a control dependency for other operators. This is only a concern in graph mode. Below is an example of how to ensure tf.print executes in graph mode:\n",
      "\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias size 384          slice_size 384          Shape[heads=12, buckets=32]                                 \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/final_layer_norm/scale                               size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 384          slice_size 384          Shape[heads=12, buckets=32]                                 \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/final_layer_norm/scale                               size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 24674304     slice_size 24674304     Shape[vocab=32128, d_model=768]                             \n",
      "INFO:tensorflow:Trainable Variables            count: 257     Total size: 222903552        Total slice_size: 222903552      \n",
      "INFO:tensorflow:All Variables                  count: 257     Total size: 222903552        Total slice_size: 222903552      \n",
      "INFO:tensorflow:Counters:\n",
      "allconcat: 2.05e+03\n",
      " allconcat/1: 2.05e+03\n",
      "  allconcat/1/reshape_op: 2.05e+03\n",
      "allreduce: 4.88e+07\n",
      " allreduce/[0]: 4.88e+07\n",
      "  allreduce/[0]/einsum_op: 4.88e+07\n",
      "  allreduce/[0]/reduce_op: 2\n",
      " allreduce/[1]: 1\n",
      "  allreduce/[1]/reduce_op: 1\n",
      "einsum: 3.47e+11\n",
      "einsum_unique: 3.47e+11\n",
      "output: 6.27e+09\n",
      " output/AddOperation: 1.77e+09\n",
      " output/BinaryOpWithBroadcasting: 6.29e+07\n",
      " output/Constant: 1.89e+07\n",
      " output/EinsumOperation: 1.23e+09\n",
      " output/ImportOperation: 1.11e+03\n",
      " output/MinMaxOperation: 3.77e+07\n",
      " output/OneHotOperation: 8.71e+08\n",
      " output/RangeOperation: 2.05e+03\n",
      " output/ReduceOperation: 9.48e+05\n",
      " output/ReshapeOperation: 1.33e+08\n",
      " output/ScalarAddOperation: 5.04e+07\n",
      " output/ScalarMultiplyOperation: 1.14e+08\n",
      " output/ShiftOperation: 1.02e+03\n",
      " output/SlicewiseOperation: 1.28e+09\n",
      " output/StopGradient: 4.53e+08\n",
      " output/Variable: 2.23e+08\n",
      " output/WhileLoopOperation: 1.89e+07\n",
      "output_unique: 6.27e+09\n",
      " output_unique/AddOperation: 1.77e+09\n",
      " output_unique/BinaryOpWithBroadcasting: 6.29e+07\n",
      " output_unique/Constant: 1.89e+07\n",
      " output_unique/EinsumOperation: 1.23e+09\n",
      " output_unique/ImportOperation: 1.11e+03\n",
      " output_unique/MinMaxOperation: 3.77e+07\n",
      " output_unique/OneHotOperation: 8.71e+08\n",
      " output_unique/RangeOperation: 2.05e+03\n",
      " output_unique/ReduceOperation: 9.48e+05\n",
      " output_unique/ReshapeOperation: 1.33e+08\n",
      " output_unique/ScalarAddOperation: 5.04e+07\n",
      " output_unique/ScalarMultiplyOperation: 1.14e+08\n",
      " output_unique/ShiftOperation: 1.02e+03\n",
      " output_unique/SlicewiseOperation: 1.28e+09\n",
      " output_unique/StopGradient: 4.53e+08\n",
      " output_unique/Variable: 2.23e+08\n",
      " output_unique/WhileLoopOperation: 1.89e+07\n",
      "variables: 2.23e+08\n",
      " variables/trainable: 2.23e+08\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/array_ops.py:1475: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from t5-base-summary/model.ckpt-537600\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Before copy master to slices.\n",
      "INFO:tensorflow:Done with copy master to slices.\n",
      "INFO:tensorflow:decoded 0: b'karangan: 1. Dr M perlu dikekalkan sebagai perdana menteri. 2. Muhyiddin perlulah menolong Dr M. 3. rakyat perlu menolong Muhyiddin.'\n",
      "INFO:tensorflow:            -> b': Presiden Parti Pribumi Bersatu Malaysia (Bersatu) Tan Sri Muhyiddin Yassin perlu memastikan Perdana Menteri, Tun Dr Mahathir Mohamad terus dikekalkan sebagai perdana menteri.Muhyiddin berkata, Muhyiddin perlu memastikan Presiden Bersatu itu kekal sebagai perdana menteri walaupun ada pihak yang menuduh beliau tidak melakukan kesalahan.\"Saya fikir, Dr Mahathir buat tuduhan yang pelbagai kepada beliau... dia buat macam-macam untuk dia (Dr Mahathir) tengok,\" katanya kepada pemberita di sini hari ini.Beliau berkata demikian semasa diminta mengulas mengenai kritikan seorang netizen terhadap Dr Mahathir yang mendakwa beliau tidak melakukan kesalahan mengikut Akta Hasutan 1948.Baru-Baru ini, seorang individu bernama Abu Bakar Al-Yamani membuat laporan polis terhadap Dr Mahathir yang juga Pengerusi Perdana Menteri.Pada Selasa, Pengerusi Pakatan Harapan (PH), Tun Dr Mahathir Mohamad berkata, beliau tidak akan campur tangan dalam urusan negara.Menurutnya, isu yang melibatkan Dr Mahathir dan Muhyiddin adalah baik kerana ia melibatkan kepentingan negara.'\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "INFO:tensorflow:prediction_loop marked as finished\n"
     ]
    }
   ],
   "source": [
    "model.predict(\n",
    "    input_file='test.txt',\n",
    "    output_file='out.txt',\n",
    "    temperature=0.7,\n",
    "    beam_size=1,\n",
    "    sentencepiece_model_path='sp10m.cased.t5.model'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf output/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_model_dir': 't5-base-summary', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "isolate_session_state: true\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': None, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver.TPUClusterResolver object at 0x7fc9483795c0>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running infer on CPU\n",
      "INFO:tensorflow:feature inputs : Tensor(\"Reshape:0\", shape=(1, 1, 1024), dtype=int32)\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias size 384          slice_size 384          Shape[heads=12, buckets=32]                                 \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/final_layer_norm/scale                               size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 384          slice_size 384          Shape[heads=12, buckets=32]                                 \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/final_layer_norm/scale                               size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 24674304     slice_size 24674304     Shape[vocab=32128, d_model=768]                             \n",
      "INFO:tensorflow:Trainable Variables            count: 257     Total size: 222903552        Total slice_size: 222903552      \n",
      "INFO:tensorflow:All Variables                  count: 257     Total size: 222903552        Total slice_size: 222903552      \n",
      "INFO:tensorflow:Counters:\n",
      "einsum: 3.47e+11\n",
      "einsum_unique: 3.47e+11\n",
      "output: 6.27e+09\n",
      " output/AddOperation: 1.77e+09\n",
      " output/BinaryOpWithBroadcasting: 6.29e+07\n",
      " output/Constant: 1.89e+07\n",
      " output/EinsumOperation: 1.23e+09\n",
      " output/ImportOperation: 1.11e+03\n",
      " output/MinMaxOperation: 3.77e+07\n",
      " output/OneHotOperation: 8.71e+08\n",
      " output/RangeOperation: 2.05e+03\n",
      " output/ReduceOperation: 9.48e+05\n",
      " output/ReshapeOperation: 1.33e+08\n",
      " output/ScalarAddOperation: 5.04e+07\n",
      " output/ScalarMultiplyOperation: 1.14e+08\n",
      " output/ShiftOperation: 1.02e+03\n",
      " output/SlicewiseOperation: 1.28e+09\n",
      " output/StopGradient: 4.53e+08\n",
      " output/Variable: 2.23e+08\n",
      " output/WhileLoopOperation: 1.89e+07\n",
      "output_unique: 6.27e+09\n",
      " output_unique/AddOperation: 1.77e+09\n",
      " output_unique/BinaryOpWithBroadcasting: 6.29e+07\n",
      " output_unique/Constant: 1.89e+07\n",
      " output_unique/EinsumOperation: 1.23e+09\n",
      " output_unique/ImportOperation: 1.11e+03\n",
      " output_unique/MinMaxOperation: 3.77e+07\n",
      " output_unique/OneHotOperation: 8.71e+08\n",
      " output_unique/RangeOperation: 2.05e+03\n",
      " output_unique/ReduceOperation: 9.48e+05\n",
      " output_unique/ReshapeOperation: 1.33e+08\n",
      " output_unique/ScalarAddOperation: 5.04e+07\n",
      " output_unique/ScalarMultiplyOperation: 1.14e+08\n",
      " output_unique/ShiftOperation: 1.02e+03\n",
      " output_unique/SlicewiseOperation: 1.28e+09\n",
      " output_unique/StopGradient: 4.53e+08\n",
      " output_unique/Variable: 2.23e+08\n",
      " output_unique/WhileLoopOperation: 1.89e+07\n",
      "variables: 2.23e+08\n",
      " variables/trainable: 2.23e+08\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from t5-base-summary/model.ckpt-537600\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: output/temp-b'1590369228'/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "model.batch_size = 1\n",
    "saved_model_path = model.export(\n",
    "    'output',\n",
    "    checkpoint_step=-1,\n",
    "    beam_size=1,\n",
    "    temperature=0.8,\n",
    "    sentencepiece_model_path='sp10m.cased.t5.model'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'output/1590369228'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saved_model_path.decode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from output/1590369228/variables/variables\n"
     ]
    }
   ],
   "source": [
    "import tensorflow_text  \n",
    "\n",
    "tf.compat.v1.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "meta_graph_def = tf.compat.v1.saved_model.load(sess, [\"serve\"], saved_model_path.decode())\n",
    "signature_def = meta_graph_def.signature_def[\"serving_default\"]\n",
    "pred = lambda x: sess.run(\n",
    "    fetches=signature_def.outputs[\"outputs\"].name, \n",
    "    feed_dict={signature_def.inputs[\"input\"].name: x}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "': Presiden Bersatu, Tan Sri Muhyiddin Yassin perlu mengekalkan Tun Dr Mahathir Mohamad sebagai perdana menteri berbanding Datuk Seri Anwar Ibrahim yang hanya minta bantuan untuk menyelesaikan kemelut kedudukan negara.Muhyiddin berkata, ini kerana semua pihak tahu masalah yang dihadapi oleh Perdana Menteri adalah di luar bidang kuasa beliau sendiri.Katanya, Muhyiddin perlu membantu beliau kerana beliau percaya rakyat Malaysia tahu apa yang berlaku di luar bidang kuasa beliau.\"Apa yang berlaku di luar bidang kuasa Dr Mahathir... semua tahu bahawa ini berlaku di bawah kepimpinan Anwar.\"Muhyiddin dan seluruh rakyat yang tahu apa yang berlaku di Johor.\"Ini kerana di Johor ini, majoriti menteri-menteri dalam Pakatan Harapan banyak sangat ketua-ketua parti.\"Jadi Muhyiddin perlu bantu Dr Mahathir sebab rakyat tahu apa yang berlaku di Johor Bahru,\" katanya dalam satu kenyataan di sini, pada Jumaat.Dalam pada itu, Muhyiddin berkata, rakyat juga perlu menolong Muhyiddin untuk menyelesaikan masalah yang melanda negara ketika ini.Menurutnya, Muhyiddin perlu menggalas tugas dengan baik dan memastikan keadaan negara berada dalam keadaan baik.'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q = f'karangan: {string}'\n",
    "\n",
    "pred([q])[0].decode('utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Shah Alam - Masyarakat juga perlu membantu kerajaan dalam menangani masalah politik dan rasuah dalam negara. Menurut Timbalan Menteri Perumahan dan Kerajaan Tempatan, Datuk Raja Kamarul Bahrin Shah Raja Ahmad, pengurusan kerajaan sepatutnya melibatkan diri dalam menyelesaikan masalah yang dibangkitkan oleh masyarakat. Katanya, semua pihak yang terlibat harus memastikan kerja-kerja pentadbiran dilaksanakan mengikut undang-undang yang ditetapkan. \"Kalau kita lihat dalam beberapa hari ini banyak unsur-unsur politik yang melibatkan salah guna kuasa, rasuah dan salah guna kuasa berlaku dalam negara kita. \"Ini kerana masalah rasuah menjadi antara isu politik yang masih berlaku ketika ini. \"Oleh itu, kita kena bantu. Sudah tentu kajian dan perancangan perlu dilakukan untuk selesaikan masalah ini,\" katanya kepada Sinar Harian. Beliau berkata demikian sebagai mengulas kenyataan Menteri Kewangan, Lim Guan Eng baru-baru ini yang meminta kerajaan meringankan beban ditanggung rakyat akibat kegiatan rasuah dalam negara. Raja Bahrin berkata, beliau berharap semua pihak tidak membiarkan keadaan dalam negara itu menjadi isu dan memberi perhatian kepada kebajikan rakyat. \"Kerajaan serta rakyat perlu memainkan peranan untuk membantu kerajaan. \"Jangan buat sesuatu perkara yang boleh memudaratkan rakyat. Jika peniaga tidak mengambil kesempatan, tolonglah orang. \"Jika tidak membantu pihak kerajaan, kita tidak akan gembira hati mereka,\" katanya. Dalam pada itu, ibu bapa perlu memastikan anak menolong kerajaan dan memberi bantuan sekiranya mereka tidak dapat membantu. Katanya, anak-anak mereka tidak mampu ini kerana semasa zaman mereka kecil membantu kerajaan di luar dari segi duit juga mahal. \"Arab Saudi yang banyak membantu dengan duit sendiri, kerajaan pula tidak memberikan bantuan. Kita bantu dengan duit sendiri. \"Kita akan bantu dalam hal kewangan sebab kedudukan duit kita memang tinggi. Apabila kita menolong kerajaan, kita akan bantu untuk anak-anak kita,\" katanya.'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = '1. kerajaan perlu tolong gotong royong. 2. masyarakat juga perlu menolong kerajaan. 3. ibu bapa perlu memastikan anak menolong kerajaan'\n",
    "q = f'karangan: {string}'\n",
    "\n",
    "pred([q])[0].decode('utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Shah Alam: Pelakon, Neelofa tetap dengan keputusan untuk berkahwin akhir tahun ini dengan berkahwin dengan Long Tiger yang menjadi teman baiknya. Neelofa berkata, dia tidak akan berputus asa kerana tiada apa yang perlu dibimbangkan. \"Tak apalah, saya yang akan uruskan semuanya. Tapi biarlah proses itu berjalan dengan baik. Selagi tak selesai, saya tak akan melarikan diri. \"Tapi sebagai suami, saya tetap dengan keputusan untuk berkahwin. Tapi saya tak akan kecewa kerana tiada apa yang perlu dibimbangkan. \"Jika saya berkahwin dengan Long Tiger, antarabangsa yang akan menahan gosip seumpama ini, saya tidak akan halang. \"Sebab itu saya tidak mahu timbul isu yang boleh mengganggu perkahwinan,\" katanya kepada Bh Online. Baru-Baru ini, kecoh apabila Long Tiger bergaduh dengan Datuk Husein Awang (Datuk Afdlin Shauki). Dalam perbincangan dengan suami, pelakon itu mendedahkan dia sudah meminta kebenaran supaya dia berkahwin dengan gadis pilihan di Singapura iaitu Hairul Azreen. Bagaimanapun, perbincangan itu dileraikan Omar dan suami, Datuk Hazwani Ahmad yang juga pelakon. Pada awalnya, suspek akan \\'tikam\\' kepala dua beradik ini dan bergaduh sesama sendiri. Namun, usaha mereka untuk meleraikan pergeseran itu gagal selepas bercerai. Omar dan Neelofa pada penghujung November lalu juga bersetuju untuk berkahwin dengan gadis yang dikenali sebagai Nora Danish itu.'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = '1. Neelofa tetap dengan keputusan untuk berkahwin akhir tahun ini. 2. Long Tiger sanggup membantu Neelofa. 3. Tiba-tiba Long Tiger bergaduh dengan Husein.'\n",
    "q = f'ringkasan: {string}'\n",
    "\n",
    "pred([q])[0].decode('utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.system(f'mv {saved_model_path.decode()} model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model/\n",
      "model/saved_model.pb\n",
      "model/variables/\n",
      "model/variables/variables.data-00000-of-00002\n",
      "model/variables/variables.data-00001-of-00002\n",
      "model/variables/variables.index\n"
     ]
    }
   ],
   "source": [
    "!tar -czvf sample-generator-t5-base.tar.gz model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "\n",
    "bucketName = 'huseinhouse-storage'\n",
    "Key = 'sample-generator-t5-base.tar.gz'\n",
    "outPutname = \"v35/generator/sample-generator-t5-base.tar.gz\"\n",
    "\n",
    "s3 = boto3.client('s3')\n",
    "s3.upload_file(Key,bucketName,outPutname)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
